from transformers import pipeline
from transformers import  AutoTokenizer
from transformers import  AutoModel
import os
def f1():
    print('1')
    from transformers import BertTokenizer

    checkpoint = BertTokenizer.from_pretrained("bert-base-uncased")
    checkpoint.tokenize("I have a new GPU!")





if __name__ == '__main__':
    os.environ['HF_TOKEN']='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'
    # 1、Tokenizer分词：将输入的文字转换为向量
    # 2、model
    # 3、post procession
    f1()


